{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import os\n",
    "\n",
    "import torch\n",
    "\n",
    "from scipy.io import loadmat\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device('cuda:0' if use_cuda else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add new methods here.\n",
    "methods = ['sift_d2net_os8_512kpts', 'sift_d2net_os4_512kpts', 'sift_d2net_os2_512kpts', 'sift_d2net_os1_512kpts', 'sift_apd2net_os4_512kpts_dils_3', 'sift_apd2net_os4_512kpts_dils_5', 'sift_apd2net_os4_512kpts_dils_7', 'sift_apd2net_os4_512kpts_dils_11', 'sift_apd2net_os4_512kpts_dils_21', 'sift_apd2net_os4_512kpts_dils_31', 'sift_apd2net_os2_512kpts_dils_3_21', 'sift_apd2net_os2_512kpts_dils_5_21', 'sift_apd2net_os2_512kpts_dils_7_21', 'sift_apd2net_os2_512kpts_dils_11_21', 'sift_apd2net_os2_512kpts_dils_15_21', 'sift_apd2net_os2_512kpts_dils_21_21', 'sift_apd2net_os2_512kpts_dils_27_27', 'sift_apd2net_os2_512kpts_dils_31_31', 'sift_apd2net_os2_512kpts_dils_37_37', 'sift_apd2net_os2_512kpts_dils_11_37', 'sift_apd2net_os2_512kpts_dils_15_37', 'sift_apd2net_os2_512kpts_dils_17_37', 'sift_apd2net_os2_512kpts_dils_19_37', 'sift_apd2net_os2_512kpts_dils_21_37', 'sift_apd2net_os2_512kpts_dils_27_37', 'sift_apd2net_os2_512kpts_dils_19_31', 'sift_apd2net_os2_512kpts_dils_19_33', 'sift_apd2net_os2_512kpts_dils_19_35', 'sift_apd2net_os1_512kpts_dils_3_19_35', 'sift_apd2net_os1_512kpts_dils_5_19_35', 'sift_apd2net_os1_512kpts_dils_9_19_35', 'sift_apd2net_os1_512kpts_dils_11_19_35', 'sift_apd2net_os1_512kpts_dils_15_19_35', 'sift_apd2net_os1_512kpts_dils_19_19_35', 'sift_apd2net_os1_512kpts_dils_19_35_35', 'sift_apd2net_os1_512kpts_dils_19_41_41', 'sift_apd2net_os1_512kpts_dils_19_51_51', 'sift_apd2net_os1_512kpts_dils_25_51_51']\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change here if you want to use top K or all features.\n",
    "# top_k = 2000\n",
    "top_k = None "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_i = 52\n",
    "n_v = 56"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = 'hpatches-sequences-release'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "lim = [1, 15]\n",
    "rng = np.arange(lim[0], lim[1] + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mnn_matcher(descriptors_a, descriptors_b):\n",
    "    device = descriptors_a.device\n",
    "    sim = descriptors_a @ descriptors_b.t()\n",
    "    nn12 = torch.max(sim, dim=1)[1]\n",
    "    nn21 = torch.max(sim, dim=0)[1]\n",
    "    ids1 = torch.arange(0, sim.shape[0], device=device)\n",
    "    mask = (ids1 == nn21[nn12])\n",
    "    matches = torch.stack([ids1[mask], nn12[mask]])\n",
    "    return matches.t().data.cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def benchmark_features(read_feats):\n",
    "    seq_names = sorted(os.listdir(dataset_path))\n",
    "\n",
    "    n_feats = []\n",
    "    n_matches = []\n",
    "    seq_type = []\n",
    "    i_err = {thr: 0 for thr in rng}\n",
    "    v_err = {thr: 0 for thr in rng}\n",
    "\n",
    "    for seq_idx, seq_name in tqdm(enumerate(seq_names), total=len(seq_names)):\n",
    "        keypoints_a, descriptors_a = read_feats(seq_name, 1)\n",
    "        n_feats.append(keypoints_a.shape[0])\n",
    "\n",
    "        for im_idx in range(2, 7):\n",
    "            keypoints_b, descriptors_b = read_feats(seq_name, im_idx)\n",
    "            n_feats.append(keypoints_b.shape[0])\n",
    "\n",
    "            matches = mnn_matcher(\n",
    "                torch.from_numpy(descriptors_a).to(device=device), \n",
    "                torch.from_numpy(descriptors_b).to(device=device)\n",
    "            )\n",
    "            \n",
    "            homography = np.loadtxt(os.path.join(dataset_path, seq_name, \"H_1_\" + str(im_idx)))\n",
    "            \n",
    "            pos_a = keypoints_a[matches[:, 0], : 2] \n",
    "            pos_a_h = np.concatenate([pos_a, np.ones([matches.shape[0], 1])], axis=1)\n",
    "            pos_b_proj_h = np.transpose(np.dot(homography, np.transpose(pos_a_h)))\n",
    "            pos_b_proj = pos_b_proj_h[:, : 2] / pos_b_proj_h[:, 2 :]\n",
    "\n",
    "            pos_b = keypoints_b[matches[:, 1], : 2]\n",
    "\n",
    "            dist = np.sqrt(np.sum((pos_b - pos_b_proj) ** 2, axis=1))\n",
    "\n",
    "            n_matches.append(matches.shape[0])\n",
    "            seq_type.append(seq_name[0])\n",
    "            \n",
    "            if dist.shape[0] == 0:\n",
    "                dist = np.array([float(\"inf\")])\n",
    "            \n",
    "            for thr in rng:\n",
    "                if seq_name[0] == 'i':\n",
    "                    i_err[thr] += np.mean(dist <= thr)\n",
    "                else:\n",
    "                    v_err[thr] += np.mean(dist <= thr)\n",
    "    \n",
    "    seq_type = np.array(seq_type)\n",
    "    n_feats = np.array(n_feats)\n",
    "    n_matches = np.array(n_matches)\n",
    "    \n",
    "    return i_err, v_err, [seq_type, n_feats, n_matches]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def summary(stats):\n",
    "    seq_type, n_feats, n_matches = stats\n",
    "    print('# Features: {:f} - [{:d}, {:d}]'.format(np.mean(n_feats), np.min(n_feats), np.max(n_feats)))\n",
    "    print('# Matches: Overall {:f}, Illumination {:f}, Viewpoint {:f}'.format(\n",
    "        np.sum(n_matches) / ((n_i + n_v) * 5), \n",
    "        np.sum(n_matches[seq_type == 'i']) / (n_i * 5), \n",
    "        np.sum(n_matches[seq_type == 'v']) / (n_v * 5))\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_read_function(method, extension='ppm'):\n",
    "    def read_function(seq_name, im_idx):\n",
    "        aux = np.load(os.path.join(dataset_path, seq_name, '%d.%s.%s' % (im_idx, extension, method)))\n",
    "        if top_k is None:\n",
    "            return aux['keypoints'], aux['descriptors']\n",
    "        else:\n",
    "            assert('scores' in aux)\n",
    "            ids = np.argsort(aux['scores'])[-top_k :]\n",
    "            return aux['keypoints'][ids, :], aux['descriptors'][ids, :]\n",
    "    return read_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sift_to_rootsift(descriptors):\n",
    "    return np.sqrt(descriptors / np.expand_dims(np.sum(np.abs(descriptors), axis=1), axis=1) + 1e-16)\n",
    "def parse_mat(mat):\n",
    "    keypoints = mat['keypoints'][:, : 2]\n",
    "    raw_descriptors = mat['descriptors']\n",
    "    l2_norm_descriptors = raw_descriptors / np.expand_dims(np.sum(raw_descriptors ** 2, axis=1), axis=1)\n",
    "    descriptors = sift_to_rootsift(l2_norm_descriptors)\n",
    "    if top_k is None:\n",
    "        return keypoints, descriptors\n",
    "    else:\n",
    "        assert('scores' in mat)\n",
    "        ids = np.argsort(mat['scores'][0])[-top_k :]\n",
    "        return keypoints[ids, :], descriptors[ids, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "if top_k is None:\n",
    "    cache_dir = 'cache'\n",
    "else:\n",
    "    cache_dir = 'cache-top'\n",
    "if not os.path.isdir(cache_dir):\n",
    "    os.mkdir(cache_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sift_d2net_os8_512kpts\n",
      "cache/sift_d2net_os8_512kpts.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 132.968519, Illumination 156.134615, Viewpoint 111.457143\n",
      "sift_d2net_os4_512kpts\n",
      "cache/sift_d2net_os4_512kpts.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 136.772222, Illumination 158.942308, Viewpoint 116.185714\n",
      "sift_d2net_os2_512kpts\n",
      "cache/sift_d2net_os2_512kpts.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 138.209259, Illumination 158.642308, Viewpoint 119.235714\n",
      "sift_d2net_os1_512kpts\n",
      "cache/sift_d2net_os1_512kpts.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 137.135185, Illumination 158.196154, Viewpoint 117.578571\n",
      "sift_apd2net_os4_512kpts_dils_3\n",
      "cache/sift_apd2net_os4_512kpts_dils_3.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 131.472222, Illumination 155.157692, Viewpoint 109.478571\n",
      "sift_apd2net_os4_512kpts_dils_5\n",
      "cache/sift_apd2net_os4_512kpts_dils_5.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 131.085185, Illumination 155.265385, Viewpoint 108.632143\n",
      "sift_apd2net_os4_512kpts_dils_7\n",
      "cache/sift_apd2net_os4_512kpts_dils_7.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 130.220370, Illumination 154.934615, Viewpoint 107.271429\n",
      "sift_apd2net_os4_512kpts_dils_11\n",
      "cache/sift_apd2net_os4_512kpts_dils_11.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 129.238889, Illumination 153.861538, Viewpoint 106.375000\n",
      "sift_apd2net_os4_512kpts_dils_21\n",
      "cache/sift_apd2net_os4_512kpts_dils_21.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 132.692593, Illumination 158.042308, Viewpoint 109.153571\n",
      "sift_apd2net_os4_512kpts_dils_31\n",
      "cache/sift_apd2net_os4_512kpts_dils_31.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 136.118519, Illumination 160.257692, Viewpoint 113.703571\n",
      "sift_apd2net_os2_512kpts_dils_3_21\n",
      "cache/sift_apd2net_os2_512kpts_dils_3_21.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 127.992593, Illumination 152.526923, Viewpoint 105.210714\n",
      "sift_apd2net_os2_512kpts_dils_5_21\n",
      "cache/sift_apd2net_os2_512kpts_dils_5_21.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 122.535185, Illumination 146.192308, Viewpoint 100.567857\n",
      "sift_apd2net_os2_512kpts_dils_7_21\n",
      "cache/sift_apd2net_os2_512kpts_dils_7_21.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 118.775926, Illumination 142.376923, Viewpoint 96.860714\n",
      "sift_apd2net_os2_512kpts_dils_11_21\n",
      "cache/sift_apd2net_os2_512kpts_dils_11_21.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 112.329630, Illumination 135.688462, Viewpoint 90.639286\n",
      "sift_apd2net_os2_512kpts_dils_15_21\n",
      "cache/sift_apd2net_os2_512kpts_dils_15_21.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 109.294444, Illumination 133.273077, Viewpoint 87.028571\n",
      "sift_apd2net_os2_512kpts_dils_21_21\n",
      "cache/sift_apd2net_os2_512kpts_dils_21_21.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 107.177778, Illumination 131.850000, Viewpoint 84.267857\n",
      "sift_apd2net_os2_512kpts_dils_27_27\n",
      "cache/sift_apd2net_os2_512kpts_dils_27_27.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 112.344444, Illumination 139.426923, Viewpoint 87.196429\n",
      "sift_apd2net_os2_512kpts_dils_31_31\n",
      "cache/sift_apd2net_os2_512kpts_dils_31_31.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 118.820370, Illumination 146.996154, Viewpoint 92.657143\n",
      "sift_apd2net_os2_512kpts_dils_37_37\n",
      "cache/sift_apd2net_os2_512kpts_dils_37_37.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 127.683333, Illumination 154.584615, Viewpoint 102.703571\n",
      "sift_apd2net_os2_512kpts_dils_11_37\n",
      "cache/sift_apd2net_os2_512kpts_dils_11_37.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 125.103704, Illumination 147.723077, Viewpoint 104.100000\n",
      "sift_apd2net_os2_512kpts_dils_15_37\n",
      "cache/sift_apd2net_os2_512kpts_dils_15_37.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 124.183333, Illumination 148.269231, Viewpoint 101.817857\n",
      "sift_apd2net_os2_512kpts_dils_17_37\n",
      "cache/sift_apd2net_os2_512kpts_dils_17_37.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 124.744444, Illumination 148.915385, Viewpoint 102.300000\n",
      "sift_apd2net_os2_512kpts_dils_19_37\n",
      "cache/sift_apd2net_os2_512kpts_dils_19_37.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 124.748148, Illumination 149.253846, Viewpoint 101.992857\n",
      "sift_apd2net_os2_512kpts_dils_21_37\n",
      "cache/sift_apd2net_os2_512kpts_dils_21_37.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 124.612963, Illumination 149.800000, Viewpoint 101.225000\n",
      "sift_apd2net_os2_512kpts_dils_27_37\n",
      "cache/sift_apd2net_os2_512kpts_dils_27_37.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 125.244444, Illumination 151.488462, Viewpoint 100.875000\n",
      "sift_apd2net_os2_512kpts_dils_19_31\n",
      "cache/sift_apd2net_os2_512kpts_dils_19_31.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 118.231481, Illumination 144.030769, Viewpoint 94.275000\n",
      "sift_apd2net_os2_512kpts_dils_19_33\n",
      "cache/sift_apd2net_os2_512kpts_dils_19_33.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 121.603704, Illumination 146.780769, Viewpoint 98.225000\n",
      "sift_apd2net_os2_512kpts_dils_19_35\n",
      "cache/sift_apd2net_os2_512kpts_dils_19_35.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 123.346296, Illumination 148.015385, Viewpoint 100.439286\n",
      "sift_apd2net_os1_512kpts_dils_3_19_35\n",
      "cache/sift_apd2net_os1_512kpts_dils_3_19_35.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 119.450000, Illumination 143.084615, Viewpoint 97.503571\n",
      "sift_apd2net_os1_512kpts_dils_5_19_35\n",
      "cache/sift_apd2net_os1_512kpts_dils_5_19_35.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 116.788889, Illumination 139.630769, Viewpoint 95.578571\n",
      "sift_apd2net_os1_512kpts_dils_9_19_35\n",
      "cache/sift_apd2net_os1_512kpts_dils_9_19_35.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 111.457407, Illumination 133.342308, Viewpoint 91.135714\n",
      "sift_apd2net_os1_512kpts_dils_11_19_35\n",
      "cache/sift_apd2net_os1_512kpts_dils_11_19_35.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 110.074074, Illumination 131.603846, Viewpoint 90.082143\n",
      "sift_apd2net_os1_512kpts_dils_15_19_35\n",
      "cache/sift_apd2net_os1_512kpts_dils_15_19_35.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 107.151852, Illumination 128.569231, Viewpoint 87.264286\n",
      "sift_apd2net_os1_512kpts_dils_19_19_35\n",
      "cache/sift_apd2net_os1_512kpts_dils_19_19_35.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 105.722222, Illumination 127.265385, Viewpoint 85.717857\n",
      "sift_apd2net_os1_512kpts_dils_19_35_35\n",
      "cache/sift_apd2net_os1_512kpts_dils_19_35_35.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 108.374074, Illumination 135.215385, Viewpoint 83.450000\n",
      "sift_apd2net_os1_512kpts_dils_19_41_41\n",
      "cache/sift_apd2net_os1_512kpts_dils_19_41_41.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 120.700000, Illumination 146.330769, Viewpoint 96.900000\n",
      "sift_apd2net_os1_512kpts_dils_19_51_51\n",
      "cache/sift_apd2net_os1_512kpts_dils_19_51_51.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 134.651852, Illumination 155.923077, Viewpoint 114.900000\n",
      "sift_apd2net_os1_512kpts_dils_25_51_51\n",
      "cache/sift_apd2net_os1_512kpts_dils_25_51_51.npy\n",
      "Loading precomputed errors...\n",
      "# Features: 499.279321 - [147, 512]\n",
      "# Matches: Overall 135.481481, Illumination 157.400000, Viewpoint 115.128571\n"
     ]
    }
   ],
   "source": [
    "for method in methods:\n",
    "    output_file = os.path.join(cache_dir, method + '.npy')\n",
    "    print(method)\n",
    "    if method == 'hesaff':\n",
    "        read_function = lambda seq_name, im_idx: parse_mat(loadmat(os.path.join(dataset_path, seq_name, '%d.ppm.hesaff' % im_idx), appendmat=False))\n",
    "    else:\n",
    "        if method == 'delf' or method == 'delf-new':\n",
    "            read_function = generate_read_function(method, extension='png')\n",
    "        else:\n",
    "            read_function = generate_read_function(method)\n",
    "    print(output_file)\n",
    "    if os.path.exists(output_file):\n",
    "        print('Loading precomputed errors...')\n",
    "        errors[method] = np.load(output_file, allow_pickle=True)\n",
    "    else:\n",
    "        errors[method] = benchmark_features(read_function)\n",
    "        np.save(output_file, errors[method])\n",
    "    summary(errors[method][-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sift_d2net_os8_512kpts\n",
      "0.5499676578993848\n",
      "sift_d2net_os4_512kpts\n",
      "0.6467065231410095\n",
      "sift_d2net_os2_512kpts\n",
      "0.6699172353219817\n",
      "sift_d2net_os1_512kpts\n",
      "0.6765393005101755\n",
      "sift_apd2net_os4_512kpts_dils_3\n",
      "0.5563377962786727\n",
      "sift_apd2net_os4_512kpts_dils_5\n",
      "0.574983823697742\n",
      "sift_apd2net_os4_512kpts_dils_7\n",
      "0.5856896197752974\n",
      "sift_apd2net_os4_512kpts_dils_11\n",
      "0.6084090837225523\n",
      "sift_apd2net_os4_512kpts_dils_21\n",
      "0.6372205807383049\n",
      "sift_apd2net_os4_512kpts_dils_31\n",
      "0.6463382116020037\n",
      "sift_apd2net_os2_512kpts_dils_3_21\n",
      "0.6364869574780929\n",
      "sift_apd2net_os2_512kpts_dils_5_21\n",
      "0.6359311171845183\n",
      "sift_apd2net_os2_512kpts_dils_7_21\n",
      "0.6357537805549773\n",
      "sift_apd2net_os2_512kpts_dils_11_21\n",
      "0.6384298143079048\n",
      "sift_apd2net_os2_512kpts_dils_15_21\n",
      "0.6425601664442725\n",
      "sift_apd2net_os2_512kpts_dils_21_21\n",
      "0.6468800436460583\n",
      "sift_apd2net_os2_512kpts_dils_27_27\n",
      "0.6575674400143419\n",
      "sift_apd2net_os2_512kpts_dils_31_31\n",
      "0.6632574299625779\n",
      "sift_apd2net_os2_512kpts_dils_37_37\n",
      "0.666815353544364\n",
      "sift_apd2net_os2_512kpts_dils_11_37\n",
      "0.6488358048640928\n",
      "sift_apd2net_os2_512kpts_dils_15_37\n",
      "0.655389235636225\n",
      "sift_apd2net_os2_512kpts_dils_17_37\n",
      "0.659963944356267\n",
      "sift_apd2net_os2_512kpts_dils_19_37\n",
      "0.662928933609649\n",
      "sift_apd2net_os2_512kpts_dils_21_37\n",
      "0.6620801820265778\n",
      "sift_apd2net_os2_512kpts_dils_27_37\n",
      "0.6682241547963614\n",
      "sift_apd2net_os2_512kpts_dils_19_31\n",
      "0.6570104962064919\n",
      "sift_apd2net_os2_512kpts_dils_19_33\n",
      "0.6595700955878532\n",
      "sift_apd2net_os2_512kpts_dils_19_35\n",
      "0.6626556837669223\n",
      "sift_apd2net_os1_512kpts_dils_3_19_35\n",
      "0.6605109794952282\n",
      "sift_apd2net_os1_512kpts_dils_5_19_35\n",
      "0.6574636630696682\n",
      "sift_apd2net_os1_512kpts_dils_9_19_35\n",
      "0.6537976148811118\n",
      "sift_apd2net_os1_512kpts_dils_11_19_35\n",
      "0.6517986154144878\n",
      "sift_apd2net_os1_512kpts_dils_15_19_35\n",
      "0.6529914615457139\n",
      "sift_apd2net_os1_512kpts_dils_19_19_35\n",
      "0.6528285815119774\n",
      "sift_apd2net_os1_512kpts_dils_19_35_35\n",
      "0.6645377806325339\n",
      "sift_apd2net_os1_512kpts_dils_19_41_41\n",
      "0.6742634144162178\n",
      "sift_apd2net_os1_512kpts_dils_19_51_51\n",
      "0.6755406685906266\n",
      "sift_apd2net_os1_512kpts_dils_25_51_51\n",
      "0.6781346261450008\n"
     ]
    }
   ],
   "source": [
    "for method in methods:\n",
    "    i_err, v_err, _ = errors[method]\n",
    "    print(method)\n",
    "    print((i_err[3] + v_err[3]) / ((n_i + n_v) * 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
